;;; -*- Mode: Lisp; Package: CCL; Coding: utf-8; -*-

(chapter "Streams"
  (defsection "Stream External Format"
    "The function {function stream-external-format} can be applied to
    (and may return a non-null result for) open streams that are not
    {code file-stream} objects.

    The function {function (setf stream-external-format)} can be used
    to change the external format of open streams created with
    {function open} or {function make-socket}.")

  (defsection "Additional keywords for {code open} and {code make-socket}"
    (para
     "{function open} and {function make-socket} have each been extended to
      take the additional keyword arguments {code :class}, {code
      :sharing}, and {code :basic}.")

    (listing :definition
      (item "{code :class}" =>
        "A symbol that names the desired class of the stream.
	The specified class must inherit from {code file-stream} for
        {function open}.")
      (item "{code :sharing}" =>
        "Specifies how a stream can be used by multiple threads.
         The possible values are: {code :private}, {code :lock} and
         {code :external}.  The default is {code :private}.  The value
         {code nil} is also accepted as a synonym for {code :external}."
        (listing :definition
          (item "{code :private}" =>
            "Specifies that the stream can only be accessed by the thread that
	     first tries to do I/O to it; that thread becomes the
	     “owner” of the stream and is not necessarily the same
	     thread as the one which created the stream.  This is the
	     default.

             There has been some discussion about the idea of
	     “transferring ownership” of a stream, but this has not
	     yet been implemented.  Attempts to do I/O on a stream
	     with {code :private} sharing from a thread other than the
	     stream's owner yield an error.")
          (item "{code :lock}" =>
            "Specifies that all access to the stream require the
	     calling thread to obtain a lock. There are separate
	     “read” and “write” locks for IO streams.  This makes it
	     possible for instance, for one thread to read from such a
	     stream while another thread writes to it.  See also
             {function make-read-write-lock}, {macro with-read-lock}, and
             {macro with-write-lock}.")
          (item "{code :external} or {code nil}" => "Specifies that I/O primitives enforce no access
		protocol.  This may be appropriate for some types of
		application which can control stream access via
		application-level protocols.  Note that since even the
		act of reading from a stream changes its internal
		state (and simultaneous access from multiple threads
		can therefore lead to corruption of that state), some
		care must be taken in the design of such protocols.")))
      (item "{code :basic}" =>
        "A boolean that indicates whether or not the stream is a Gray stream,
	 i.e., whether or not the stream is an instance of {code
	 fundamental-stream} or {code ccl::basic-stream} (see {section
	 Basic Versus Fundamental Streams}).  Defaults to {code t}.")))

  (defsection "Basic Versus Fundamental Streams"
    "Gray streams (see
     {section Creating Your Own Stream Classes with Gray Streams})
      all inherit from {code fundamental-stream} whereas
     basic streams inherit from {code ccl::basic-stream}.  The
     tradeoff between {code fundamental} and {code basic} streams is
     entirely between flexibility and performance, potential or
     actual.  I/O primitives can recognize {code basic-stream}s and
     exploit knowledge of implementation details.  On the other hand,
     {code fundamental} stream classes can be subclassed and extended
     in a standard way (using the Gray streams protocol).

     For existing stream classes ({code file-stream}s, {code socket}s,
     and the internal {code ccl::fd-stream} classes used to implement
     file streams and sockets), a lot of code can be shared between
     the {code fundamental} and {code basic} implementations.  The
     biggest difference should be that that code can be reached from
     I/O primitives like {function read-char} without going through
     some steps that are there to support generality and
     extensibility, and skipping those steps when that support is not
     needed can improve I/O performance.
      
     The Gray stream method {function stream-read-char} should work on
     appropriate {code basic-stream}s.  (There may still be cases
     where such methods are undefined; such cases should be considered
     bugs.)  It is not guaranteed that Gray stream methods would ever
     be called by I/O primitives to read a character from a {code
     basic-stream}, though there are still cases where this happens.

     A simple loop reading 2M characters from a text file runs about
     10X faster when the file is opened the new defaults
     {code (:sharing :private :basic t)} than it had before these
     changes were made.  That sounds good, until one realizes that
     the “equivalent” C loop can be about 10X faster still...")

  (defsection "Stream Timeouts and Deadlines"
    (para "A stream that is associated with a file descriptor has
        attributes and accessors:
        {function stream-input-timeout},
        {function stream-output-timeout}, and
        {function stream-deadline}.  All three accessors have
        corresponding {function setf} methods.
        {function stream-input-timeout} and
        {function stream-output-timeout} are specified in
        seconds and can be any positive real number less than one million.
        When a timeout is set and the corresponding I/O operation takes
        longer than the specified interval, an error is signalled.  The
        error is {code INPUT-TIMEOUT} for input and
        {code OUTPUT-TIMEOUT} for output.
        {code STREAM-DEADLINE} specifies an absolute time in
        internal-time-units.  If an I/O operation on the stream does not
        complete before the deadline then a
        {code COMMUNICATION-DEADLINE-EXPIRED} error is
        signalled.  A deadline takes precedence over any
        input/output timeouts that may be set."))

  (defsection "Open File Streams"
    (para "{CCL} maintains a list of open file streams.  This helps
        to ensure that streams get closed in an orderly manner when
        the lisp exits.  The following thread-safe functions manage
        this list.")

    (definition (:function open-file-streams) "open-file-streams" nil
      "Returns a fresh list of file streams that are currently open.")

    (definition (:function note-open-file-stream)
	"note-open-file-stream file-stream" nil
      "Adds {param file-stream} to the internal list of open file
         streams that is returned by {function open-file-streams}.
         This function is thread-safe.  It will usually only be called
         from custom stream code when a file-stream is created.")

    (definition (:function remove-open-file-stream)
	"remove-open-file-stream file-stream" nil
      "Removes {param file-stream} from the internal list of open file
         streams that is returned by {function open-file-streams}.
         This function is thread-safe.  It will usually only be called
         from custom stream code when a file-stream is closed."))

  (defsection "Auto-flushing Streams"
    "{CCL} supports a mechanism for automatically flushing (by calling
     {function force-output}) a set of user-specified output streams.

     This is implemented by a mostly-idle thread that wakes up about 3
     times per second.

     Certain system-defined streams, such as the output side of
     {variable *terminal-io*}, are auto-flushed by default. "

    (definition (:function add-auto-flush-stream)
	"add-auto-flush-stream output-stream" nil
      "Adds {param output-stream} to a list of streams whose buffered output
       should be peridically flushed.")  

    (definition (:function remove-auto-flush-stream)
	"remove-auto-flush-stream output-stream" nil
      "Remove {param output-stream} from the list of automatically-flushed
       streams."))

  (defsection "Creating Your Own Stream Classes with Gray Streams"
    (defsection "Overview"
      "This section is still being written and revised, because
        it is woefully incomplete.  The dictionary section currently
        only lists a couple functions.  Caveat lector.

        Gray streams are an extension to Common Lisp.  They were
        proposed for standardization by David Gray (the astute reader
        now understands their name) quite some years ago, but not
        accepted, because they had not been tried sufficiently to find
        conceptual problems with them.

        They have since been implemented by quite a few modern
        Lisp implementations.  However, they do indeed have some
        inadequacies, and each implementation has addressed these in
        different ways.  The situation today is that it's difficult to
        even find out how to get started using Gray streams.  This is
        why standards are important.

        Here's a list of some classes which you might wish for
        your new stream class to inherit from:"
      (listing :column
        (item "fundamental-stream")
        (item "fundamental-input-stream")
        (item "fundamental-output-stream")
        (item "fundamental-character-stream")
        (item "fundamental-binary-stream")
        (item "fundamental-character-input-stream")
        (item "fundamental-character-output-stream")
        (item "fundamental-binary-input-stream")
        (item "fundamental-binary-output-stream")
        (item "ccl::buffered-stream-mixin")
        (item "ccl::buffered-input-stream-mixin")
        (item "ccl::buffered-output-stream-mixin")
        (item "ccl::buffered-io-stream-mixin")
        (item "ccl::buffered-character-input-stream-mixin")
        (item "ccl::buffered-character-output-stream-mixin")
        (item "ccl::buffered-character-io-stream-mixin")
        (item "ccl::buffered-binary-input-stream-mixin")
        (item "ccl::buffered-binary-output-stream-mixin")
        (item "ccl::buffered-binary-io-stream-mixin")
        (item "file-stream")
        (item "file-input-stream")
        (item "file-output-stream")
        (item "file-io-stream")
        (item "file-character-input-stream")
        (item "file-character-output-stream")
        (item "file-character-io-stream")
        (item "file-binary-input-stream")
        (item "file-binary-output-stream")
        (item "file-binary-io-stream")
        (item "ccl::fd-stream")
        (item "ccl::fd-input-stream")
        (item "ccl::fd-output-stream")
        (item "ccl::fd-io-stream")
        (item "ccl::fd-character-input-stream")
        (item "ccl::fd-character-output-stream")
        (item "ccl::fd-character-io-stream")
        (item "ccl::fd-binary-input-stream")
        (item "ccl::fd-binary-output-stream")
        (item "ccl::fd-binary-io-stream"))
      #:|All of these are defined in ccl/level-1/l1-streams.lisp,
        except for the ccl:file-* ones, which are in
        ccl/level-1/l1-sysio.lisp.

        According to the original Gray streams proposal, you
        should inherit from the most specific of the fundamental-*
        classes which applies.  Using {CCL}, though, if you want
        buffering for better performance, which, unless you know of some
        reason you wouldn't, you do, you should instead inherit from the
        appropriate ccl::buffered-* class The buffering you get this way
        is exactly the same as the buffering which is used on ordinary,
        non-Gray streams, and force-output will work properly on
        it.

        Notice that -mixin suffix in the names of all the
        ccl::buffered-* classes?  The suffix means that this class is
        not "complete" by itself; you still need to inherit from a
        fundamental-* stream, even if you also inherit from a *-mixin
        stream.  You might consider making your own class like this.
        ....  Except that they do inherit from the fundamental-*
        streams, that's weird.

        If you want to be able to create an instance of your class
        with the :class argument to (open) and (with-open-file), you
        should make it inherit from one of the file-* classes.  If you
        do this, it's not necessary to inherit from any of the other
        classes (though it won't hurt anything), since the file-*
        classes already do.

        When you inherit from the file-* classes, you can use
        (call-next-method) in any of your methods to get the standard
        behavior.  This is especially useful if you want to create a
        class which performs some simple filtering operation, such as
        changing everything to uppercase or to a different character
        encoding.  If you do this, you will definitely need to
        specialize ccl::select-stream-class.  Your method on
        ccl::stream-select-class should accept an instance of the class,
        but pay no attention to its contents, and return a symbol naming
        the class to actually be instantiated.

        If you need to make your functionality generic across all
        the different types of stream, probably the best way to
        implement it is to make it a mixin, define classes with all the
        variants of input, output, io, character, and binary, which
        inherit both from your mixin and from the appropriate other
        class, then define a method on ccl::select-stream-class which
        chooses from among those classes.

        Note that some of these classes are internal to the CCL
        package.  If you try to inherit from those ones without the
        ccl:: prefix, you'll get an error which may confuse you, calling
        them "forward-referenced classes".  That just means you used the
        wrong symbol, so add the prefix.

        Here's a list of some generic functions which you might
        wish to specialize for your new stream class, and which ought to
        be documented at some point.|
      (listing :column
        (item "stream-direction stream =>")
        (item "stream-device stream direction =>")
        (item "stream-length stream {code &optional} new =>")
        (item "stream-position stream {code &optional} new =>")
        (item "streamp stream => boolean")
        (item "stream-write-char output-stream char =>")
        (item "stream-write-entire-string output-stream string =>")
        (item "stream-read-char input-stream =>")
        (item "stream-unread-char input-stream char =>")
        (item "stream-force-output output-stream => nil")
        (item "stream-maybe-force-output output-stream => nil")
        (item "stream-finish-output output-stream => nil")
        (item "stream-clear-output output-stream => nil")
        (item "close stream {code &key} abort => boolean")
        (item "stream-fresh-line stream => t")
        (item "stream-line-length stream => length")
        (item "interactive-stream-p stream => boolean")
        (item "stream-clear-input input-stream => nil")
        (item "stream-listen input-stream => boolean")
        (item "stream-filename stream => string")
        (item "ccl::select-stream-class instance in-p out-p char-p =>
	      class"))
      (para "The following functions are standard parts of Common Lisp, but
        behave in special ways with regard to Gray streams.")
      (listing :column
        (item "open-stream-p stream => generalized-boolean")
        (item "input-stream-p stream => generalized-boolean")
        (item "output-stream-p stream => generalized-boolean")
        (item "stream-element-type stream =>")
        (item "stream-error-stream =>")
        (item "open")
        (item "close")
        (item "with-open-file"))
      "Specifically, (open) and (with-open-file) accept a new
        keyword argument, :class, which may be a symbol naming a class;
        the class itself; or an instance of it.  The class so given must
        be a subtype of 'stream, and an instance of it with no
        particular contents will be passed to ccl::select-stream-class
        to determine what class to actually instantiate.

        The following are standard, and do not behave specially
        with regard to Gray streams, but probably should."
      (listing :column (item "stream-external-format")))
    (defsection "Extending READ-SEQUENCE and WRITE-SEQUENCE"
      (defsection "Overview"
        #:|The "Gray Streams" API is based on an informal proposal that was
	      made before ANSI CL adopted the READ-SEQUENCE and WRITE-SEQUENCE
	      functions; as such, there is no "standard" way for the author of a Gray
	      stream class to improve the performance of these functions by exploiting
	      knowledge of the stream's internals (e.g., the buffering mechanism it
	      uses.)

	      In the absence of any such knowledge, READ-SEQUENCE and
	      WRITE-SEQUENCE are effectively just convenient shorthand for a
	      loop which calls READ-CHAR/READ-BYTE/WRITE-CHAR/WRITE-BYTE as
	      appropriate. The mechanism described below allows subclasses
	      of FUNDAMENTAL-STREAM to define more specialized (and
	      presumably more efficient) behavior.|)
      (defsection "Notes"
        (para "READ-SEQUENCE and WRITE-SEQUENCE do a certain amount of
	      sanity-checking and normalization of their arguments before
	      dispatching to one of the methods above. If an individual
	      method can't do anything particularly clever, CALL-NEXT-METHOD
	      can be used to handle the general case."))
      (defsection "Example"
        (code-block "
(defclass my-string-input-stream (fundamental-character-input-stream)
  ((string :initarg :string :accessor my-string-input-stream-string)
   (index :initform 0 :accessor my-string-input-stream-index)
   (length)))

(defmethod stream-read-vector ((stream my-string-input-stream) vector start end)
  (if (not (typep vector 'simple-base-string))
      (call-next-method)
      (with-slots (string index length)
	      (do* ((outpos start (1+ outpos)))
               ((or (= outpos end)
                    (= index length))
                outpos))
        (setf (schar vector outpos)
              (schar string index))
        (incf index)))))
	    ")))
    (defsection "Multibyte I/O"
      #:|All heap-allocated objects in {CCL} that cannot contain
        pointers to lisp objects are represented as
        {emphasis ivectors}. {CCL} provides low-level
        functions, and , to efficiently transfer data between buffered
        streams and ivectors. There's some overlap in functionality
        between the functions described here and the ANSI CL
        READ-SEQUENCE and WRITE-SEQUENCE functions.

        As used here, the term "octet" means roughly the same
        thing as the term "8-bit byte". The functions described below
        transfer a specified sequence of octets between a buffered
        stream and an ivector, and don't really concern themselves with
        higher-level issues (like whether that octet sequence is within
        bounds or how it relates to the logical contents of the
        ivector.) For these reasons, these functions are generally less
        safe and more flexible than their ANSI counterparts.|)
    (defsection "Gray Streams Dictionary"

      (definition (:generic-function stream-read-list) "stream-read-list stream list count" nil
        (para "Should try to read up to count elements from stream
	        into the list {param list}, returning the number of elements
	        actually read (which may be less than count in case of
	        a premature end-of-file.)")
        (defsection "Arguments and Values"
          (listing :definition
                   (item "{param stream}" ccldoc::=> "a stream, presumably a fundamental-input-stream.")
                   (item "{param list}" ccldoc::=> "a list. When a STREAM-READ-LIST method is called by
		          READ-SEQUENCE, this argument is guaranteed to be a proper
		          list.")
                   (item "{param count}" ccldoc::=> "a non-negative integer. When a STREAM-READ-LIST method
		          is called by READ-SEQUENCE, this argument is guaranteed not
		          to be greater than the length of the list."))))

      (definition (:generic-function stream-write-list) "stream-write-list stream list count" nil
        (para "Write the first {param count} elements of {param list} to
	        {param stream}. The return value of this method is ignored.")
        (defsection "Arguments and Values"
          (listing :definition
                   (item "{param stream}" ccldoc::=> "a stream, presumably a fundamental-output-stream.")
                   (item "{param list}" ccldoc::=> "a list. When a STREAM-WRITE-LIST method is called by
		          WRITE-SEQUENCE, this argument is guaranteed to be a proper
		          list.")
                   (item "{param count}" ccldoc::=> "a non-negative integer. When a STREAM-WRITE-LIST
		          method is called by WRITE-SEQUENCE, this argument is
		          guaranteed not to be greater than the length of the list."))))

      (definition (:generic-function stream-read-vector) "stream-read-vector stream vector start end" nil
        (para "Read successive elements from stream into {param
	        vector}, starting at element {param start} (inclusive)
	        and continuing through element {param
	        end} (exclusive.) Should return the index of the
	        vector element beyond the last one stored into, which
	        may be less than {param end} in case of premature
	        end-of-file.")
        (defsection "Arguments and Values"
          (listing :definition
                   (item "{param stream}" ccldoc::=> "a stream, presumably a fundamental-input-stream")
                   (item "{param vector}" ccldoc::=> "a vector. When a STREAM-READ-VECTOR method is called
		          by {code read-sequence}, this argument is guaranteed to be a simple
		          one-dimensional array.")
                   (item "{param start}" ccldoc::=> "a non-negative integer. When a STREAM-READ-VECTOR
		          method is called by READ-SEQUENCE, this argument is
		          guaranteed to be no greater than end and not greater than
		          the length of vector.")
                   (item "{param end}" ccldoc::=> "a non-negative integer. When a STREAM-READ-VECTOR
		          method is called by READ-SEQUENCE, this argument is
		          guaranteed to be no less than end and not greater than the
		          length of vector."))))

      (definition (:generic-function stream-write-vector) "stream-write-vector stream vector start end" nil
        (para "should try to write successive elements of {param
                vector} to {param stream}, starting at element {param
                start} (inclusive) and continuing through element
                {param end} (exclusive.)")
        (defsection "Arguments and Values"
          (listing :definition
                   (item "{param stream}" ccldoc::=> "a stream, presumably a fundamental-output-stream")
                   (item "{param vector}" ccldoc::=> "a vector. When a STREAM-WRITE-VECTOR method is called
		          by WRITE-SEQUENCE, this argument is guaranteed to be a
		          simple one-dimensional array.")
                   (item "{param start}" ccldoc::=> "a non-negative integer. When a STREAM-WRITE-VECTOR
		          method is called by WRITE-SEQUENCE, this argument is
		          guaranteed to be no greater than end and not greater than
		          the length of vector.")
                   (item "{param end}" ccldoc::=> "a non-negative integer. When a STREAM-WRITE-VECTOR
		          method is called by WRITE-SEQUENCE, this argument is
		          guaranteed to be no less than end and not greater than the
		          length of vector."))))

      (definition (:generic-function stream-device) "ccl::stream-device s direction"
          "Returns the OS file descriptor associated with a
	        given lisp stream."
        (defsection "Method Signatures"
          "{code {function stream-device}
	        (s stream) direction => fd}")
        (defsection "Arguments and Values"
          (listing :definition
                   (item "{param s}" ccldoc::=> "a stream.")
                   (item "{param direction}" ccldoc::=> "either :INPUT or :OUTPUT.")
                   (item "{param fd}" ccldoc::=> "a file descriptor, which is a non-negative integer
		          used by the OS to refer to an open file, socket, or similar
		          I/O connection.  NIL if there is no file descriptor associated
		          with {param s} in the direction given by
		          {param direction}.")))
        (defsection "Description"
          (para "Returns the file descriptor associated with
	        {param s} in the direction given by
	        {param direction}.  It is necessary to specify
	        {param direction} because the input and output
	        file descriptors may be different; the most common case is when
	        one of them has been redirected by the Unix shell.")))

      (definition (:generic-function stream-read-ivector) "stream-read-ivector stream ivector start-octet max-octets"
          nil
        (defsection "Description"
          (para "Reads up to max-octets octets from stream into ivector,
	        storing them at start-octet. Returns the number of octets actually
	        read."))
        (defsection "Arguments"
          (listing :definition
                   (item "{param stream}" ccldoc::=> "An input stream. The method defined on
		          BUFFERED-INPUT-STREAMs requires that the size in octets of
		          an instance of the stream's element type is 1.")
                   (item "{param ivector}" ccldoc::=> "Any ivector.")
                   (item "{param start-octet}" ccldoc::=> "A non-negative integer.")
                   (item "{param max-octets}" ccldoc::=> "A non-negative integer. The return value may be less
		          than the value of this parameter if EOF was encountered."))))

      (definition (:generic-function stream-write-ivector) "stream-write-ivector stream ivector start-octet max-octets"
          nil
        (defsection "Description"
          (para "Writes max-octets octets to stream from ivector, starting at
	        start-octet. Returns max-octets."))
        (defsection "Arguments"
          (listing :definition
                   (item "{param stream}" ccldoc::=> "An input stream. The method defined on
		          BUFFERED-OUTPUT-STREAMs requires that the size in octets of
		          an instance of the stream's element type is 1.")
                   (item "{param ivector}" ccldoc::=> "Any ivector")
                   (item "{param start-octet}" ccldoc::=> "A non-negative integer.")
                   (item "{param max-octet}" ccldoc::=> "A non-negative integer.")))
        (defsection "Examples"
          (code-block #:|
;;; Write the contents of a (SIMPLE-ARRAY(UNSIGNED-BYTE 16) 3) 
;;; to a character file stream. Read back the characters.
(let* ((a (make-array 3 
                      :element-type '(unsigned-byte 16)
                      :initial-contents '(26725 27756 28449))))
  (with-open-file (s "junk"
                     :element-type 'character
                     :direction :io
                     :if-does-not-exist :create
                     :if-exists :supersede)
    ;; Write six octets (three elements).
    (stream-write-ivector s a 0 6)
    ;; Rewind, then read a line
    (file-position s 0)
    (read-line s)))

;;; Write a vector of DOUBLE-FLOATs. Note that (to maintain
;;; alignment) there are 4 octets of padding before the 0th 
;;; element of a (VECTOR DOUBLE-FLOAT) on 32-bit platforms.
;;; (Note that (= (- target::misc-dfloat-offset 
;;;                  target::misc-data-offset) 4))
(defun write-double-float-vector
    (stream vector &key (start 0) (end (length vector)))
     (check-type vector (vector double-float))
     (let* ((start-octet (+ (* start 8) 
                            (- target::misc-dfloat-offset
                               target::misc-data-offset)))
	        (num-octets (* 8 (- end start))))
       (stream-write-ivector stream vector start-octet num-octets)))
          |)))))

  (defsection "Lisp Standard Streams and OS Standard Streams"
    "In normal interactive usage, the input and output sides of the
     bidirectional stream {code *terminal-io*} are hooked up to the
     the operating system's standard input and standard output.  The
     lisp streams {code *standard-input*}, {code *standard-output*},
     and {code *error-output*} are synonym streams for {code
     *terminal-io*}.

     In batch mode, this arrangement is modified slightly.  The lisp
     streams {code *standard-input*}, {code *standard-output*}, and
     {code *standard-error*} correspond directly to the operating
     system's standard input, standard output, and standard error.  If
     the lisp can determine that it has access to an operating system
     tty, then {code *terminal-io*} will be hooked up to that.
     Otherwise, the input and output streams of {code *terminal-io*}
     will correspond to the operating system's standard input and
     standard output.")
) ;chapter
